from __future__ import division
from operator import itemgetter, attrgetter
from struct import *
# import gc
import copy
import math
import os
import random
import sys
import time
from sets import Set
from random import randint
import re
import numpy as np
from os import walk
# import matplotlib.pyplot as plt
from subprocess import call
import os

# For format1 (the current one):
# step1
# Updated by Wei on 20141030 night
# output the binary file for external sort, good operation
print "Begins..."
# input arguments:
# read from the /san/share
# write to the /home
# arg1: 20000 # total number of queries needed to process
# arg2: ifn # the input file we are going to parse
# arg3: ofn # the ofn file we are going to output
# arg4: ofn2 # the log file for processing this file

if len(sys.argv) != 5:
    print "USAGE: programName.py NUM_QUERY_TO_PARSE ifn ofn ofn_log"
    exit(1)
else:
    NUM_QUERY_TO_PARSE = int(sys.argv[1])
    ifn = sys.argv[2]
    ofn = sys.argv[3]
    ofn2 = sys.argv[4]
    pathElements = sys.argv[3].strip().split("/")
    # new setting
    # print pathElements
    ofn3 = pathElements[0] + "/" + pathElements[1] + "/" + pathElements[2] + "/" + pathElements[3] + "/" + "binaryFiles" + "/" + pathElements[5] + ".binary"
    # original setting
    # ofn3 = sys.argv[3] + ".binary"

queryDict = {}
queryCounter = 0
# do the docHit and postingHit one by one
# postingHitDict = {}
documentHitDict = {}

ifh = open(ifn,"r")
ofh = open(ofn,"w")
ofh2 = open(ofn2,"w")
ofh3 = open(ofn3,"w")

print "ifn:",ifn
print "ofn:",ofn
print "ofn2:",ofn2
print "ofn3:",ofn3

l = ifh.readline()
while l:
    le = l.strip().split(" ")
    
    if l.strip().startswith("Search"):
        currentSearchContent = l.strip()[8:]
        if currentSearchContent.strip() != "":
            queryCounter += 1
            if queryCounter % 1000 == 0:
                ofh2.write(str(queryCounter) + " " + str(len(documentHitDict)) + "\n")
                # print queryCounter,len(documentHitDict)
                # print "len(documentHitDict):",len(documentHitDict)
            if queryCounter == NUM_QUERY_TO_PARSE:
                for i in range(0,25205179):
                    if str(i) in documentHitDict:
                        # NOT going to write anything to the disk
                        # ofh.write(str(i) + " " + str(documentHitDict[str(i)]) + "\n")
                        # write the binary format to the disk
                        ofh3.write(pack("2I",i,documentHitDict[str(i)]))
                    else:
                        pass
                break
    
    if len(le) == 15 and le[-2].startswith("GX") and int(le[0]) <= 1000:
        currentDocID = le[-3] 
        if currentDocID not in documentHitDict:
            documentHitDict[currentDocID] = 1
        else:
            documentHitDict[currentDocID] += 1
    
    l = ifh.readline()
ifh.close()
ofh.close()
ofh2.close()
ofh3.close()

print "Overall:"
print "ifn:",ifn
print "ofn:",ofn
print "ofn2:",ofn2
print "ofn3:",ofn3
print "Ends."

'''
# For format2 (the previous one):
# step1
# Updated by Wei on 20141030 night
# output the binary file for external sort, good operation
print "Begins..."
# input arguments:
# read from the /san/share
# write to the /home
# arg1: 20000 # total number of queries needed to process
# arg2: ifn # the input file we are going to parse
# arg3: ofn # the ofn file we are going to output
# arg4: ofn2 # the log file for processing this file

if len(sys.argv) != 5:
    print "USAGE: programName.py NUM_QUERY_TO_PARSE ifn ofn ofn_log"
    exit(1)
else:
    NUM_QUERY_TO_PARSE = int(sys.argv[1])
    ifn = sys.argv[2]
    ofn = sys.argv[3]
    ofn2 = sys.argv[4]
    pathElements = sys.argv[3].strip().split("/")
    # new setting
    ofn3 = pathElements[0] + "/" + pathElements[1] + "/" + pathElements[2] + "/" + "binaryFiles" + "/" + pathElements[4] + ".binary"
    # original setting
    # ofn3 = sys.argv[3] + ".binary"

queryDict = {}
queryCounter = 0
# do the docHit and postingHit one by one
# postingHitDict = {}
documentHitDict = {}

ifh = open(ifn,"r")
ofh = open(ofn,"w")
ofh2 = open(ofn2,"w")
ofh3 = open(ofn3,"w")

l = ifh.readline()
while l:
    le = l.strip().split(" ")
    # print len(le)
    
    if l.strip().startswith("Search"):
        currentSearchContent = l.strip()[8:]
        # print "currentSearchContent:",currentSearchContent
        if currentSearchContent.strip() != "":
            queryCounter += 1
            if queryCounter % 1000 == 0:
                ofh2.write(str(queryCounter) + " " + str(len(documentHitDict)) + "\n")
                # print queryCounter,len(documentHitDict)
                # print "len(documentHitDict):",len(documentHitDict)
            if queryCounter == NUM_QUERY_TO_PARSE:
                print len(documentHitDict)
                for i in range(0,25205179):
                    if str(i) in documentHitDict:
                        # NOT going to write anything to the disk
                        # ofh.write(str(i) + " " + str(documentHitDict[str(i)]) + "\n")
                        # write the binary format to the disk
                        ofh3.write(pack("2I",i,documentHitDict[str(i)]))
                    else:
                        pass
                break
    
    if len(le) == 25 and le[-2].startswith("GX") and int(le[0]) <= 10:
        # print "in"
        currentDocID = le[-3] 
        if currentDocID not in documentHitDict:
            documentHitDict[currentDocID] = 1
        else:
            documentHitDict[currentDocID] += 1
    
    l = ifh.readline()
ifh.close()
ofh.close()
ofh2.close()
ofh3.close()

print "Overall:"
print "ifn:",ifn
print "ofn:",ofn
print "ofn2:",ofn2
print "ofn3:",ofn3
print "Ends."
'''





